﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using HtmlAgilityPack;
using DWH.Utils;
using DWH.Data;
using log4net;

namespace DWH.PP.Recursive
{
    public class PPRecursiveBoardPullTask : PPBoardPullTaskBase
    {
        private string boardUrl;// = "http://70.85.48.252/bt/thread.php?fid=14";        
        private string boardUrlRootPrefix;
        private string pageNumberFlagString;
        
        private string nextPageUrlPattern;// = @"thread.php\?fid=14&search=&page=[\d+]";

        private bool pageHistoryDetectAndRestart = false;
        private int pageHistoryDetectNumber = 1000;

        private Dictionary<int, string> visitedPageListUrls;
        private Dictionary<int, string> visitedPageContentUrls;

        private int conflictPage = 0;
        private int restartIndicator = 0;

        private ILog logger;

        public override void Initialize()
        {
            base.Initialize();

            logger = LogManager.GetLogger(typeof(PPRecursiveBoardPullTask));            

            //if ((PPTaskResumeData != null)
            //    && (PPTaskResumeData.ContainsKey(DWHConstants.KEY_BOARD_START_URL))
            //    && (!String.IsNullOrEmpty(PPTaskResumeData[DWHConstants.KEY_BOARD_START_URL])))
            //{
            //    boardUrl = PPTaskResumeData[DWHConstants.KEY_BOARD_START_URL];
            //}
            //else
            //{
            //    boardUrl = ExtendData[DWHConstants.KEY_BOARD_START_URL];
            //}

            boardUrl = ExtendData[DWHConstants.KEY_BOARD_START_URL];
            pageNumberFlagString = ExtendData[DWHConstants.KEY_PAGE_NUMBER_FLAG_STRING];

            
            nextPageUrlPattern = ExtendData[DWHConstants.KEY_NEXT_PAGE_URL_PATTERN];

            if (ExtendData.ContainsKey(DWHConstants.KEY_PAGE_HISTORY_DETECT_RESTART))
            {
                pageHistoryDetectAndRestart = Boolean.Parse(ExtendData[DWHConstants.KEY_PAGE_HISTORY_DETECT_RESTART]);
            }

            if (ExtendData.ContainsKey(DWHConstants.KEY_PAGE_HISTORY_DETECT_NUMBER))
            {
                pageHistoryDetectNumber = Int32.Parse(ExtendData[DWHConstants.KEY_PAGE_HISTORY_DETECT_NUMBER]);
            }
            else
            {
                pageHistoryDetectNumber = DWHConstants.V_PAGE_HISTORY_RESTART_DEFAULT;
            }

            boardUrlRootPrefix = boardUrl.Substring(0, boardUrl.LastIndexOf('/') + 1);

            visitedPageListUrls = new Dictionary<int, string>();
            visitedPageContentUrls = new Dictionary<int, string>();
        }

        public override void Run()
        {
            logger.InfoFormat("task {0} started", TaskName);
            visitedPageListUrls.Add(boardUrl.GetHashCode(), boardUrl);
            ProcessPageList(boardUrl);
        }

        public override void Reset()
        {
            logger.InfoFormat("reset task {0}",TaskName);
            visitedPageListUrls.Clear();
            boardUrl = ExtendData[DWHConstants.KEY_BOARD_START_URL];
            conflictPage = 0;
        }

        public override void ProcessPageList(string currentPageListUrl)
        {
            logger.InfoFormat("task {0}, processing page list:{1}", TaskName, currentPageListUrl);

            restartIndicator++;

            HtmlWeb web = new HtmlWeb();
            web.OverrideEncoding = Encoding.GetEncoding(pageEncoding);

            HtmlDocument doc = web.Load(currentPageListUrl);

            List<string> tempPageContentUrls;
            List<string> tempPageListUrls;
            tempPageContentUrls = new List<string>();
            tempPageListUrls = new List<string>();

            HtmlNodeCollection canLinks = doc.DocumentNode.SelectNodes("//a[@href]");
            foreach (HtmlNode link in canLinks)
            {
                string linkUrl = link.Attributes["href"].Value;

                //<a target=_blank href='htm_data/14/1207/536987.html'>和新來[11P]</a>
                //<a target=_blank href='http://70.85.48.252/bt/htm_data/14/1207/536987.html'>和新來[11P]</a>
                //<a target=_blank href='htm_data/14/1110/489962.html'>情[10P]</a>
                //<a target=_blank href='http://70.85.48.252/bt/htm_data/14/1110/489962.html'>情[10P]</a>
                if (DWHHelper.PatternMatch(linkUrl, contentPageUrlPattern))
                {
                    linkUrl = DWHHelper.FormatUrl(boardUrlRootPrefix, linkUrl);
                    //add page content
                    if (visitedPageContentUrls.ContainsKey(linkUrl.GetHashCode()))
                    {
                        conflictPage++;
                        logger.InfoFormat("task {0} ,this page has been processed in this run: {1}", TaskName, linkUrl);
                    }
                    else
                    {
                        if (!tempPageContentUrls.Contains(linkUrl))
                        {
                            tempPageContentUrls.Add(linkUrl);
                        }
                    }
                }

                //http://70.85.48.252/bt/thread.php?fid=14&search=&page=5
                if (DWHHelper.PatternMatch(linkUrl, nextPageUrlPattern))
                {
                    linkUrl = DWHHelper.FormatUrl(boardUrlRootPrefix, linkUrl);
                    //add next page
                    if (!visitedPageListUrls.ContainsKey(linkUrl.GetHashCode()))
                    {
                        if (!tempPageListUrls.Contains(linkUrl))
                        {
                            tempPageListUrls.Add(linkUrl);
                        }
                    }
                }
            }

            //process page content first            
            foreach (string pageContentUrl in tempPageContentUrls)
            {
                if (DWHDBHelper.IsPageVisited(pageContentUrl.GetHashCode()))
                {
                    conflictPage++;
                    logger.InfoFormat("task {0} ,this page has been processed in previous run: {1}", TaskName, pageContentUrl);
                }
                else
                {
                    //if (!DWHDBHelper.IsPageVisited(pageContentUrl.GetHashCode()) 
                    //    && !visitedPageContentUrls.ContainsKey(pageContentUrl.GetHashCode()))

                    try
                    {
                        visitedPageContentUrls.Add(pageContentUrl.GetHashCode(), pageContentUrl);
                        ProcessPageContent(pageContentUrl);                        
                    }
                    catch (Exception ex)
                    {
                        logger.ErrorFormat("Error occurs when processing pageContentUrl:{0} , {1}", pageContentUrl, ex.ToString());
                    }
                }
            }

            //update task info to database
            DWHDBHelper.AddOrUpdatePageList(TaskName, currentPageListUrl.GetHashCode(), currentPageListUrl,DateTime.Now);
            if (PPTaskResumeData != null && PPTaskResumeData.ContainsKey(DWHConstants.KEY_BOARD_START_URL))
            {
                PPTaskResumeData[DWHConstants.KEY_BOARD_START_URL] = currentPageListUrl;
            }
            else
            {
                PPTaskResumeData.Add(DWHConstants.KEY_BOARD_START_URL, currentPageListUrl);
            }
            DWHDBHelper.UpdatePPTask(TaskName, PPTaskResumeData);

            //if done, restart;
            if (pageHistoryDetectAndRestart && (conflictPage > pageHistoryDetectNumber))
            {
                logger.InfoFormat("task {0} has come to conflict page for restart, restart", TaskName);
                boardUrl = ExtendData[DWHConstants.KEY_BOARD_START_URL];
                visitedPageContentUrls.Clear();
                visitedPageListUrls.Clear();
                conflictPage = 0;
                Run();
            }            
            
            //process page list second, only the rest page           
            List<string> filteredPageListUrls = FilterPageListUrls(tempPageListUrls, currentPageListUrl);
            foreach (string pageList in filteredPageListUrls)
            {
                if (!visitedPageListUrls.ContainsKey(pageList.GetHashCode()))
                {
                    try
                    {
                        visitedPageListUrls.Add(pageList.GetHashCode(), pageList);
                        ProcessPageList(pageList);
                    }
                    catch (Exception ex)
                    {
                        logger.ErrorFormat("error occures when processing pageListUrl:{0},{1}", pageList, ex.ToString());
                    }
                }
            }

            restartIndicator--;

            if (restartIndicator == 0)
            {                
                boardUrl = ExtendData[DWHConstants.KEY_BOARD_START_URL];

                logger.InfoFormat("task {0} finished for this run, try to start over from board url:{1}", TaskName, boardUrl);

                visitedPageListUrls.Clear();
                ProcessPageList(boardUrl);
            }
        }

        //http://istorrs.com/download/?paged=99
        protected override List<string> FilterPageListUrls(List<string> originalPageListUrls, string currentPageListUrl)
        {
            List<string> baseList = base.FilterPageListUrls(originalPageListUrls, currentPageListUrl);
            List<string> result = new List<string>();

            int currentNum = getPageNumber(currentPageListUrl);

            foreach (string url in baseList)
            {
                if (getPageNumber(url) > currentNum)
                {
                    result.Add(url);
                }
            }

            result.Sort((x, y) => (-1) * String.Compare(x, y, true));

            return result;
        }

        private int getPageNumber(string url)
        {
            string flagStr = pageNumberFlagString;
            try
            {
                return Int32.Parse(url.Substring(url.IndexOf(flagStr) + flagStr.Length));
            }
            catch { }

            return 0;
        }
       
    }
}
